This is an automated email from the ASF dual-hosted git repository. michaelsmith pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 6d3317b9a11c858f412337e44b207548498d8b4f Author: Daniel Becker <[email protected]> AuthorDate: Fri Nov 17 13:50:24 2023 +0100 IMPALA-12570: Add longer strings to tables containing collections IMPALA-12373 introduces small string optimisation, after which not all strings will have a var-len part. IMPALA-12159 adds support for ORDER BY for collections of variable length types in the select list, but the test tables it uses only/mostly contain short strings. This patch has two modifications: 1. It introduces longer strings in 'collection_tbl' and 'collection_struct_mix'. It also adds two more rows to the existing one in 'collection_tbl' so that it can be used in sorting tests. These tables are only used by complex types tests, so the impact is limited. 2. It modifies RandomNestedDataGenerator.java, so that now it takes a parameter for string length. Some variable names are changed to clearer names. The references to and uses of RandomNestedDataGenerator are updated. Change-Id: Ief770d6bc9258fce159a733d5afa34fe594b96f8 Reviewed-on: http://gerrit.cloudera.org:8080/20718 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- .../datagenerator/RandomNestedDataGenerator.java | 21 ++-- testdata/bin/generate-load-nested.sh | 10 +- testdata/data/README | 2 +- .../functional/functional_schema_template.sql | 124 +++++++++++++++++---- .../QueryTest/mixed-collections-and-structs.test | 46 ++++---- .../QueryTest/nested-array-in-select-list.test | 8 +- .../QueryTest/nested-map-in-select-list.test | 28 +++-- 7 files changed, 169 insertions(+), 70 deletions(-) diff --git a/java/datagenerator/src/main/java/org/apache/impala/datagenerator/RandomNestedDataGenerator.java b/java/datagenerator/src/main/java/org/apache/impala/datagenerator/RandomNestedDataGenerator.java index fc68847c4..f77bb4ebd 100644 --- a/java/datagenerator/src/main/java/org/apache/impala/datagenerator/RandomNestedDataGenerator.java +++ b/java/datagenerator/src/main/java/org/apache/impala/datagenerator/RandomNestedDataGenerator.java @@ -42,7 +42,8 @@ import org.apache.parquet.hadoop.metadata.CompressionCodecName; public class RandomNestedDataGenerator { public static Random rand; - public static int numListItems; + public static int maxNormalStrLen; + public static int maxNumListItems; public static int numElementsGenerated = 0; public static ArrayList<Double> doubleCache; public static ArrayList<Float> floatCache; @@ -126,7 +127,7 @@ public class RandomNestedDataGenerator { stringCache = new ArrayList<String>(); for (int i = 0; i < NUM_ELEMENTS; i++) { StringBuilder sb = new StringBuilder(); - int len = rand.nextInt(10); + int len = rand.nextInt(maxNormalStrLen); for (int j = 0; j < len; j++) { sb.append(alphabet.charAt(rand.nextInt(alphabet.length()))); } @@ -144,7 +145,7 @@ public class RandomNestedDataGenerator { private static int generateListLength(int depth) { if (rand.nextDouble() < 0.1) return 0; // empty list - return rand.nextInt(numListItems); + return rand.nextInt(maxNumListItems); } private static Schema getNonNullSchema(Schema schema) { @@ -254,19 +255,21 @@ public class RandomNestedDataGenerator { public static void main(String[] args) throws Exception { final int num_args = args.length; - if (num_args < 4 || num_args > 5) { + if (num_args < 5 || num_args > 6) { System.err.println( - "Arguments: schema_file num_elements list_len output_file [random_seed]"); + "Arguments: schema_file num_elements max_normal_str_len max_list_len " + + "output_file [random_seed]"); System.exit(1); } String schemaFile = args[0]; int numElements = Integer.valueOf(args[1]); - numListItems = Integer.valueOf(args[2]); - String outputFile = args[3]; + maxNormalStrLen = Integer.valueOf(args[2]); + maxNumListItems = Integer.valueOf(args[3]); + String outputFile = args[4]; Optional<Long> seed; - if (num_args > 4) { - seed = Optional.of(Long.valueOf(args[4])); + if (num_args > 5) { + seed = Optional.of(Long.valueOf(args[5])); } else { seed = Optional.empty(); } diff --git a/testdata/bin/generate-load-nested.sh b/testdata/bin/generate-load-nested.sh index b773ed0fc..768ef7e88 100755 --- a/testdata/bin/generate-load-nested.sh +++ b/testdata/bin/generate-load-nested.sh @@ -25,16 +25,17 @@ SHELL_CMD=${IMPALA_HOME}/bin/impala-shell.sh usage() { echo "Usage: $0 [-n num databases] [-e num elements]" 1>&2; - echo " [-i num list items] [-f]" 1>&2; + echo " [-l string length] [-i num list items] [-f]" 1>&2; echo "Tables will be flattened if -f is set." 1>&2; exit 1; } NUM_DATABASES=1 NUM_ELEMENTS=$((10**7)) +STR_LEN=10 NUM_FIELDS=30 FLATTEN=false -while getopts ":n:e:i:f" OPTION; do +while getopts ":n:e:l:i:f" OPTION; do case "${OPTION}" in n) NUM_DATABASES=${OPTARG} @@ -42,6 +43,9 @@ while getopts ":n:e:i:f" OPTION; do e) NUM_ELEMENTS=${OPTARG} ;; + l) + STR_LEN=${OPTARG} + ;; i) NUM_FIELDS=${OPTARG} ;; @@ -81,7 +85,7 @@ do TABLE_NAME="${FILE_NAME%.*}" mvn $IMPALA_MAVEN_OPTIONS -f "${IMPALA_HOME}/java/datagenerator/pom.xml" exec:java \ -Dexec.mainClass="org.apache.impala.datagenerator.RandomNestedDataGenerator" \ - -Dexec.args="${AVRO_SCHEMA_PATH} ${NUM_ELEMENTS} ${NUM_FIELDS} ${DB_DIR}/${TABLE_NAME}/${TABLE_NAME}.parquet"; + -Dexec.args="${AVRO_SCHEMA_PATH} ${NUM_ELEMENTS} ${STR_LEN} ${NUM_FIELDS} ${DB_DIR}/${TABLE_NAME}/${TABLE_NAME}.parquet"; if $FLATTEN; then mvn $IMPALA_MAVEN_OPTIONS -f "${IMPALA_HOME}/java/TableFlattener/pom.xml" \ diff --git a/testdata/data/README b/testdata/data/README index c885d81bd..c08681a87 100644 --- a/testdata/data/README +++ b/testdata/data/README @@ -1007,7 +1007,7 @@ Generated with RandomNestedDataGenerator.java from the following schema: The following command was used: mvn -f "${IMPALA_HOME}/java/datagenerator/pom.xml" exec:java -Dexec.mainClass="org.apache.impala.datagenerator.RandomNestedDataGenerator" - -Dexec.args="${input_table_schema}.avsc 1500000 15 '${output_file}.parquet'"; + -Dexec.args="${input_table_schema}.avsc 1500000 10 15 '${output_file}.parquet'"; empty_present_stream.orc: Generated by ORC C++ library using the following code diff --git a/testdata/datasets/functional/functional_schema_template.sql b/testdata/datasets/functional/functional_schema_template.sql index a15e5bd28..79978062f 100644 --- a/testdata/datasets/functional/functional_schema_template.sql +++ b/testdata/datasets/functional/functional_schema_template.sql @@ -3877,27 +3877,27 @@ INSERT OVERWRITE TABLE {db_name}{db_suffix}.{table_name} VALUES array(1, 2, NULL), array(array(1, 2, NULL), array(3)), array(array(array(1, 2, NULL), array(3)), array(array(4))), - array("1", "2", NULL), - array(array("1", "2", NULL), array("3")), - array(array(array("1", "2", NULL), array("3")), array(array("4"))), - map(1, "first", 2, "second"), - map(1, map(10, "ten", 20, "twenty"), 2, map(30, "thirty", 40, "forty")), + array("1", "two wooden boxes", NULL), + array(array("one silk glove", "2", NULL), array("three pancakes")), + array(array(array("1", "second harmonic", NULL), array("three cities")), array(array("four castles"))), + map(1, "first automobile", 2, "second"), + map(1, map(10, "ten", 20, "twentieth paragraph"), 2, map(30, "thirty minutes", 40, "forty")), map( - 1, map(10, map(100, "hundred", 200, "two hundred"), 20, map(300, "three hundred", 400, "four hundred")), - 2, map(30, map(500, "five hundred", 600, "six hundred"), 40, map(700, "seven hundred", 800, "eight hundred")) + 1, map(10, map(100, "hundred", 200, "two hundred pages"), 20, map(300, "three hundred pages", 400, "four hundred")), + 2, map(30, map(500, "five hundred pages", 600, "six hundred"), 40, map(700, "seven hundred pages", 800, "eight hundred")) ), map( 1, map(10, array(100, 200), 20, array(300, 400)), 2, map(30, array(500, 600), 40, array(700, 800)) ), - map(true, "true", false, "false"), - map(-1Y, "a", 0Y, "b", 1Y, "c"), - map(-1S, "a", 0S, "b", 1S, "c"), - map(-1L, "a", 0L, "b", 1L, "c"), - map(cast(-1.5 as FLOAT), "a", cast(0.25 as FLOAT), "b", cast(1.75 as FLOAT), "c"), - map(cast(-1.5 as DOUBLE), "a", cast(0.25 as DOUBLE), "b", cast(1.75 as DOUBLE), "c"), - map(-1.8, "a", 0.2, "b", 1.2, "c"), - map("one", 1, "two", 2, "three", 3), + map(true, "true", false, "false statement"), + map(-1Y, "a nice sunny day", 0Y, "best day in my life", 1Y, "c"), + map(-1S, "a nice sunny day", 0S, "best day in my life", 1S, "c"), + map(-1L, "a nice sunny day", 0L, "best day in my life", 1L, "c"), + map(cast(-1.5 as FLOAT), "a nice sunny day", cast(0.25 as FLOAT), "best day in my life", cast(1.75 as FLOAT), "c"), + map(cast(-1.5 as DOUBLE), "a nice sunny day", cast(0.25 as DOUBLE), "best day in my life", cast(1.75 as DOUBLE), "c"), + map(-1.8, "a nice sunny day", 0.2, "best day in my life", 1.2, "c"), + map("one", 1, "two", 2, "three distinct values", 3), map(cast("Mon" as CHAR(3)), 1, cast("Tue" as CHAR(3)), 2, cast("Wed" as CHAR(3)), 3, @@ -3909,7 +3909,83 @@ INSERT OVERWRITE TABLE {db_name}{db_suffix}.{table_name} VALUES map(cast("a" as VARCHAR(3)), "A", cast("ab" as VARCHAR(3)), "AB", cast("abc" as VARCHAR(3)), "ABC"), map(to_utc_timestamp("2022-12-10 08:15:12", "UTC"), "Saturday morning", to_utc_timestamp("2022-12-09 18:15:12", "UTC"), "Friday evening"), - map(to_date("2022-12-10"), "Saturday", to_date("2022-12-09"), "Friday") + map(to_date("2022-12-10"), "Saturday 24 hours", to_date("2022-12-09"), "Friday") + ), + (2, + array(1, NULL, 3), + array(array(NULL, 1, 2, NULL), array(5, 14, NULL)), + array(array(array(NULL, 1, 2, NULL), array(5, 14, NULL)), array(array(NULL, 5))), + array("one dinosaur bone", NULL, "2", NULL), + array(array("1", "2", NULL, "four dinosaur bones"), array("five dinosaur bones")), + array(array(array("second dinosaur bone", NULL, NULL), array("three dinosaur bones")), array(array("one", NULL, "four dinosaur bones"))), + map(1, "first dinosaur bone", 2, "second", 3, NULL), + map(1, map(10, "ten dinosaur bones", 20, "20"), 2, map(30, "thirty dinosaur bones", 40, "forty dinosaur bones")), + map( + 1, map(10, map(100, "hundred", 200, "two hundred dinosaur bones"), 20, map(300, "three hundred dinosaur bones", 400, "four hundred")), + 2, map(30, map(500, "five hundred dinosaur bones", 600, "six hundred"), 40, map(700, "seven hundred dinosaur bones", 800, "eight hundred")) + ), + map( + 1, map(10, array(100, 200), 20, array(300, 400)), + 2, map(30, array(500, 600), 40, array(700, 800)) + ), + map(true, "true", false, "false dinosaur bones"), + map(-1Y, "a nice dinosaur bone", 0Y, "best dinosaur bone", 1Y, "c"), + map(-1S, "a nice dinosaur bone", 0S, "best dinosaur bone", 1S, "c"), + map(-1L, "a nice dinosaur bone", 0L, "best dinosaur bone", 1L, "c"), + map(cast(-1.5 as FLOAT), "a nice dinosaur bone", cast(0.25 as FLOAT), "best dinosaur bone", cast(1.75 as FLOAT), "c"), + map(cast(-1.5 as DOUBLE), "a nice dinosaur bone", cast(0.25 as DOUBLE), "best dinosaur bone", cast(1.75 as DOUBLE), "c"), + map(-1.8, "a nice dinosaur bone", 0.2, "best dinosaur bone", 1.2, "c"), + map("one", 1, "two", 2, "three distinct dinosaur bones", 3), + map(cast("Mon" as CHAR(3)), 1, + cast("Tue" as CHAR(3)), 2, + cast("Wed" as CHAR(3)), 3, + cast("Thu" as CHAR(3)), 4, + cast("Fri" as CHAR(3)), 5, + cast("Sat" as CHAR(3)), 6, + cast("Sun" as CHAR(3)), 7 + ), + map(cast("a" as VARCHAR(3)), "A", cast("ab" as VARCHAR(3)), "AB", cast("abc" as VARCHAR(3)), "ABC"), + map(to_utc_timestamp("2022-12-10 08:15:12", "UTC"), "Saturday morning", + to_utc_timestamp("2022-12-09 18:15:12", "UTC"), "Friday evening"), + map(to_date("2022-12-10"), "Saturday 24 dinosaur bones", to_date("2022-12-09"), "Friday") + ), + (3, + array(NULL, 4679, NULL, 49, NULL), + array(array(1, 2, NULL, NULL, 856), array(365, 855, 369, NULL)), + array(array(array(1, NULL, 2, NULL), array(NULL, 15)), array(array(NULL, 4))), + array("1", NULL, "three even-toed ungulates"), + array(array("one even-toed ungulate", "2", NULL, NULL), array(NULL, "three even-toed ungulates")), + array(array(array("1", "-1", "second even-toed ungulate", NULL), array("three even-toed ungulates")), array(array("four even-toed ungulate"))), + map(645, "fourth even-toed ungulate", 5, "fifth"), + map(1, map(10, "ten", 20, "twentieth even-toed ungulate"), 2, map(30, "thirty even-toed ungulates", 40, "forty")), + map( + 1, map(10, map(100, "hundred", 200, "two hundred even-toed ungulates"), 20, map(300, "three hundred even-toed ungulates", 400, "four hundred")), + 2, map(30, map(500, "five hundred even-toed ungulates", 600, "six hundred"), 40, map(700, "seven hundred even-toed ungulates", 800, "eight hundred")) + ), + map( + 1, map(10, array(100, 200), 20, array(300, 400)), + 2, map(30, array(500, 600), 40, array(700, 800)) + ), + map(true, "true even-toed ungulate", false, "false"), + map(-1Y, "a nice even-toed ungulate", 0Y, "best even-toed ungulate", 1Y, "c"), + map(-1S, "a nice even-toed ungulate", 0S, "best even-toed ungulate", 1S, "c"), + map(-1L, "a nice even-toed ungulate", 0L, "best even-toed ungulate", 1L, "c"), + map(cast(-1.5 as FLOAT), "a nice even-toed ungulate", cast(0.25 as FLOAT), "best even-toed ungulate", cast(1.75 as FLOAT), "c"), + map(cast(-1.5 as DOUBLE), "a nice even-toed ungulate", cast(0.25 as DOUBLE), "best even-toed ungulate", cast(1.75 as DOUBLE), "c"), + map(-1.8, "a nice even-toed ungulate", 0.2, "best even-toed ungulate", 1.2, "c"), + map("one", 1, "two", 2, "three distinct even-toed ungulates", 3), + map(cast("Mon" as CHAR(3)), 1, + cast("Tue" as CHAR(3)), 2, + cast("Wed" as CHAR(3)), 3, + cast("Thu" as CHAR(3)), 4, + cast("Fri" as CHAR(3)), 5, + cast("Sat" as CHAR(3)), 6, + cast("Sun" as CHAR(3)), 7 + ), + map(cast("a" as VARCHAR(3)), "A", cast("ab" as VARCHAR(3)), "AB", cast("abc" as VARCHAR(3)), "ABC"), + map(to_utc_timestamp("2022-12-10 08:15:12", "UTC"), "Saturday morning", + to_utc_timestamp("2022-12-09 18:15:12", "UTC"), "Friday evening"), + map(to_date("2022-12-10"), "Saturday 24 even-toed ungulates", to_date("2022-12-09"), "Friday") ); ---- LOAD ==== @@ -3992,11 +4068,11 @@ INSERT OVERWRITE TABLE {db_name}{db_suffix}.{table_name} VALUES ( 1, named_struct("arr", array(1, 2, 3, 4, NULL, NULL, 5)), - named_struct("m", map(1, "one", 2, "two", 0, NULL)), + named_struct("m", map(1, "one spaceship captain", 2, "two", 0, NULL)), array(named_struct("i", 1L), named_struct("i", 2L), named_struct("i", 3L), named_struct("i", 4L), NULL, named_struct("i", 5L), named_struct("i", NULL)), array(named_struct("inner_struct", named_struct("str", "", "l", 0), "small", 2S), NULL, - named_struct("inner_struct", named_struct("str", "some_string", "l", 5), "small", 20S)), + named_struct("inner_struct", named_struct("str", "some spaceship captain", "l", 5), "small", 20S)), named_struct("arr", array(array(to_date("2022-12-05"), to_date("2022-12-06"), NULL, to_date("2022-12-07")), array(to_date("2022-12-08"), to_date("2022-12-09"), NULL)), "i", 2), map( @@ -4016,7 +4092,7 @@ INSERT OVERWRITE TABLE {db_name}{db_suffix}.{table_name} VALUES "n", 98 ), "small", named_struct( - "str", "somestring", + "str", "a few spaceship captains", "i", 100 ) ) @@ -4025,11 +4101,11 @@ INSERT OVERWRITE TABLE {db_name}{db_suffix}.{table_name} VALUES ( 2, named_struct("arr", if(false, array(1), NULL)), - named_struct("m", if(false, map(1, "one"), NULL)), + named_struct("m", if(false, map(1, "one soju distillery"), NULL)), array(named_struct("i", 100L), named_struct("i", 8L), named_struct("i", 35L), named_struct("i", 45L), NULL, named_struct("i", 193L), named_struct("i", NULL)), array(named_struct("inner_struct", if(false, named_struct("str", "", "l", 0), NULL), "small", 104S), - named_struct("inner_struct", named_struct("str", "aaa", "l", 28), "small", 105S), NULL), + named_struct("inner_struct", named_struct("str", "a few soju distilleries", "l", 28), "small", 105S), NULL), named_struct("arr", array(array(to_date("2022-12-10"), to_date("2022-12-11"), NULL, to_date("2022-12-12")), if(false, array(to_date("2022-12-12")), NULL)), "i", 2754), map( @@ -4049,7 +4125,7 @@ INSERT OVERWRITE TABLE {db_name}{db_suffix}.{table_name} VALUES "n", 95 ), "small", named_struct( - "str", "otherstring", + "str", "other soju distillery", "i", 2048 ) ), @@ -4065,7 +4141,7 @@ INSERT OVERWRITE TABLE {db_name}{db_suffix}.{table_name} VALUES "n", 8 ), "small", named_struct( - "str", "textstring", + "str", "test soju distillery", "i", 0 ) ), @@ -4081,7 +4157,7 @@ INSERT OVERWRITE TABLE {db_name}{db_suffix}.{table_name} VALUES "n", 93 ), NULL), "small", named_struct( - "str", "nextstring", + "str", "next soju distillery", "i", 128 ) ), diff --git a/testdata/workloads/functional-query/queries/QueryTest/mixed-collections-and-structs.test b/testdata/workloads/functional-query/queries/QueryTest/mixed-collections-and-structs.test index 56ba1c5d1..56175127a 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/mixed-collections-and-structs.test +++ b/testdata/workloads/functional-query/queries/QueryTest/mixed-collections-and-structs.test @@ -11,8 +11,8 @@ select all_mix from collection_struct_mix; ---- RESULTS -1,'{"arr":[1,2,3,4,null,null,5]}','{"m":{1:"one",2:"two",0:null}}','[{"i":1},{"i":2},{"i":3},{"i":4},null,{"i":5},{"i":null}]','[{"inner_struct":{"str":"","l":0},"small":2},null,{"inner_struct":{"str":"some_string","l":5},"small":20}]','{"arr":[["2022-12-05","2022-12-06",null,"2022-12-07"],["2022-12-08","2022-12-09",null]],"i":2}','{10:{"big":{"arr":[{"inner_arr":[[0,null,-1,-5,null,8],[20,null]],"m":"2022-12-05 14:30:00"},{"inner_arr":[[12,1024,null],[null,null,84],[null,15,null]],"m":" [...] -2,'{"arr":null}','{"m":null}','[{"i":100},{"i":8},{"i":35},{"i":45},null,{"i":193},{"i":null}]','[{"inner_struct":null,"small":104},{"inner_struct":{"str":"aaa","l":28},"small":105},null]','{"arr":[["2022-12-10","2022-12-11",null,"2022-12-12"],null],"i":2754}','{20:{"big":{"arr":[null,{"inner_arr":[[12,1024,null],[null,null,84],[null,15,null]],"m":"2022-12-10 08:15:12"}],"n":95},"small":{"str":"otherstring","i":2048}},21:{"big":{"arr":null,"n":8},"small":{"str":"textstring","i":0}},22:{" [...] +1,'{"arr":[1,2,3,4,null,null,5]}','{"m":{1:"one spaceship captain",2:"two",0:null}}','[{"i":1},{"i":2},{"i":3},{"i":4},null,{"i":5},{"i":null}]','[{"inner_struct":{"str":"","l":0},"small":2},null,{"inner_struct":{"str":"some spaceship captain","l":5},"small":20}]','{"arr":[["2022-12-05","2022-12-06",null,"2022-12-07"],["2022-12-08","2022-12-09",null]],"i":2}','{10:{"big":{"arr":[{"inner_arr":[[0,null,-1,-5,null,8],[20,null]],"m":"2022-12-05 14:30:00"},{"inner_arr":[[12,1024,null],[null,n [...] +2,'{"arr":null}','{"m":null}','[{"i":100},{"i":8},{"i":35},{"i":45},null,{"i":193},{"i":null}]','[{"inner_struct":null,"small":104},{"inner_struct":{"str":"a few soju distilleries","l":28},"small":105},null]','{"arr":[["2022-12-10","2022-12-11",null,"2022-12-12"],null],"i":2754}','{20:{"big":{"arr":[null,{"inner_arr":[[12,1024,null],[null,null,84],[null,15,null]],"m":"2022-12-10 08:15:12"}],"n":95},"small":{"str":"other soju distillery","i":2048}},21:{"big":{"arr":null,"n":8},"small":{"s [...] ---- TYPES INT,STRING,STRING,STRING,STRING,STRING,STRING ==== @@ -141,9 +141,9 @@ select id, item from collection_struct_mix, collection_struct_mix.arr_contains_n ---- RESULTS 1,'{"inner_struct":{"str":"","l":0},"small":2}' 1,'NULL' -1,'{"inner_struct":{"str":"some_string","l":5},"small":20}' +1,'{"inner_struct":{"str":"some spaceship captain","l":5},"small":20}' 2,'{"inner_struct":null,"small":104}' -2,'{"inner_struct":{"str":"aaa","l":28},"small":105}' +2,'{"inner_struct":{"str":"a few soju distilleries","l":28},"small":105}' 2,'NULL' ---- TYPES INT,STRING @@ -157,9 +157,9 @@ select id, item from sub2, sub2.arr2 a; ---- RESULTS 1,'{"inner_struct":{"str":"","l":0},"small":2}' 1,'NULL' -1,'{"inner_struct":{"str":"some_string","l":5},"small":20}' +1,'{"inner_struct":{"str":"some spaceship captain","l":5},"small":20}' 2,'{"inner_struct":null,"small":104}' -2,'{"inner_struct":{"str":"aaa","l":28},"small":105}' +2,'{"inner_struct":{"str":"a few soju distilleries","l":28},"small":105}' 2,'NULL' ---- TYPES INT,STRING @@ -172,9 +172,9 @@ select id, item from collection_struct_mix_view, ---- RESULTS 1,'{"inner_struct":{"str":"","l":0},"small":2}' 1,'NULL' -1,'{"inner_struct":{"str":"some_string","l":5},"small":20}' +1,'{"inner_struct":{"str":"some spaceship captain","l":5},"small":20}' 2,'{"inner_struct":null,"small":104}' -2,'{"inner_struct":{"str":"aaa","l":28},"small":105}' +2,'{"inner_struct":{"str":"a few soju distilleries","l":28},"small":105}' 2,'NULL' ---- TYPES INT,STRING @@ -229,9 +229,9 @@ select id, a.item, a.item.inner_struct, a.item.small from collection_struct_mix, ---- RESULTS 1,'{"inner_struct":{"str":"","l":0},"small":2}','{"str":"","l":0}',2 1,'NULL','NULL',NULL -1,'{"inner_struct":{"str":"some_string","l":5},"small":20}','{"str":"some_string","l":5}',20 +1,'{"inner_struct":{"str":"some spaceship captain","l":5},"small":20}','{"str":"some spaceship captain","l":5}',20 2,'{"inner_struct":null,"small":104}','NULL',104 -2,'{"inner_struct":{"str":"aaa","l":28},"small":105}','{"str":"aaa","l":28}',105 +2,'{"inner_struct":{"str":"a few soju distilleries","l":28},"small":105}','{"str":"a few soju distilleries","l":28}',105 2,'NULL','NULL',NULL ---- TYPES INT,STRING,STRING,SMALLINT @@ -243,9 +243,9 @@ select id, a.item, a.item.inner_struct, a.item.small from collection_struct_mix_ ---- RESULTS 1,'{"inner_struct":{"str":"","l":0},"small":2}','{"str":"","l":0}',2 1,'NULL','NULL',NULL -1,'{"inner_struct":{"str":"some_string","l":5},"small":20}','{"str":"some_string","l":5}',20 +1,'{"inner_struct":{"str":"some spaceship captain","l":5},"small":20}','{"str":"some spaceship captain","l":5}',20 2,'{"inner_struct":null,"small":104}','NULL',104 -2,'{"inner_struct":{"str":"aaa","l":28},"small":105}','{"str":"aaa","l":28}',105 +2,'{"inner_struct":{"str":"a few soju distilleries","l":28},"small":105}','{"str":"a few soju distilleries","l":28}',105 2,'NULL','NULL',NULL ---- TYPES INT,STRING,STRING,SMALLINT @@ -258,9 +258,9 @@ select id, a.item, a.item.inner_struct, a.item.small from sub, ---- RESULTS 1,'{"inner_struct":{"str":"","l":0},"small":2}','{"str":"","l":0}',2 1,'NULL','NULL',NULL -1,'{"inner_struct":{"str":"some_string","l":5},"small":20}','{"str":"some_string","l":5}',20 +1,'{"inner_struct":{"str":"some spaceship captain","l":5},"small":20}','{"str":"some spaceship captain","l":5}',20 2,'{"inner_struct":null,"small":104}','NULL',104 -2,'{"inner_struct":{"str":"aaa","l":28},"small":105}','{"str":"aaa","l":28}',105 +2,'{"inner_struct":{"str":"a few soju distilleries","l":28},"small":105}','{"str":"a few soju distilleries","l":28}',105 2,'NULL','NULL',NULL ---- TYPES INT,STRING,STRING,SMALLINT @@ -345,7 +345,7 @@ STRING set STRINGIFY_MAP_KEYS=1; select id, struct_contains_map from collection_struct_mix; ---- RESULTS -1,'{"m":{"1":"one","2":"two","0":null}}' +1,'{"m":{"1":"one spaceship captain","2":"two","0":null}}' 2,'{"m":null}' ---- TYPES INT,STRING @@ -478,8 +478,8 @@ select * from ( select * from sub ) sub2; ---- RESULTS -1,'{"arr":[1,2,3,4,null,null,5]}','{"m":{1:"one",2:"two",0:null}}','[{"i":1},{"i":2},{"i":3},{"i":4},null,{"i":5},{"i":null}]','[{"inner_struct":{"str":"","l":0},"small":2},null,{"inner_struct":{"str":"some_string","l":5},"small":20}]','{"arr":[["2022-12-05","2022-12-06",null,"2022-12-07"],["2022-12-08","2022-12-09",null]],"i":2}','{10:{"big":{"arr":[{"inner_arr":[[0,null,-1,-5,null,8],[20,null]],"m":"2022-12-05 14:30:00"},{"inner_arr":[[12,1024,null],[null,null,84],[null,15,null]],"m":" [...] -2,'{"arr":null}','{"m":null}','[{"i":100},{"i":8},{"i":35},{"i":45},null,{"i":193},{"i":null}]','[{"inner_struct":null,"small":104},{"inner_struct":{"str":"aaa","l":28},"small":105},null]','{"arr":[["2022-12-10","2022-12-11",null,"2022-12-12"],null],"i":2754}','{20:{"big":{"arr":[null,{"inner_arr":[[12,1024,null],[null,null,84],[null,15,null]],"m":"2022-12-10 08:15:12"}],"n":95},"small":{"str":"otherstring","i":2048}},21:{"big":{"arr":null,"n":8},"small":{"str":"textstring","i":0}},22:{" [...] +1,'{"arr":[1,2,3,4,null,null,5]}','{"m":{1:"one spaceship captain",2:"two",0:null}}','[{"i":1},{"i":2},{"i":3},{"i":4},null,{"i":5},{"i":null}]','[{"inner_struct":{"str":"","l":0},"small":2},null,{"inner_struct":{"str":"some spaceship captain","l":5},"small":20}]','{"arr":[["2022-12-05","2022-12-06",null,"2022-12-07"],["2022-12-08","2022-12-09",null]],"i":2}','{10:{"big":{"arr":[{"inner_arr":[[0,null,-1,-5,null,8],[20,null]],"m":"2022-12-05 14:30:00"},{"inner_arr":[[12,1024,null],[null,n [...] +2,'{"arr":null}','{"m":null}','[{"i":100},{"i":8},{"i":35},{"i":45},null,{"i":193},{"i":null}]','[{"inner_struct":null,"small":104},{"inner_struct":{"str":"a few soju distilleries","l":28},"small":105},null]','{"arr":[["2022-12-10","2022-12-11",null,"2022-12-12"],null],"i":2754}','{20:{"big":{"arr":[null,{"inner_arr":[[12,1024,null],[null,null,84],[null,15,null]],"m":"2022-12-10 08:15:12"}],"n":95},"small":{"str":"other soju distillery","i":2048}},21:{"big":{"arr":null,"n":8},"small":{"s [...] ---- TYPES INT,STRING,STRING,STRING,STRING,STRING,STRING ==== @@ -494,8 +494,8 @@ select id, arr_contains_nested_struct, struct_contains_nested_arr.* from ( select id, arr_contains_nested_struct, struct_contains_nested_arr from sub ) sub2; ---- RESULTS -1,'[{"inner_struct":{"str":"","l":0},"small":2},null,{"inner_struct":{"str":"some_string","l":5},"small":20}]',2 -2,'[{"inner_struct":null,"small":104},{"inner_struct":{"str":"aaa","l":28},"small":105},null]',2754 +1,'[{"inner_struct":{"str":"","l":0},"small":2},null,{"inner_struct":{"str":"some spaceship captain","l":5},"small":20}]',2 +2,'[{"inner_struct":null,"small":104},{"inner_struct":{"str":"a few soju distilleries","l":28},"small":105},null]',2754 ---- TYPES INT,STRING,INT ==== @@ -510,8 +510,8 @@ select id, arr_contains_nested_struct, struct_contains_nested_arr.* from ( select id, arr_contains_nested_struct, struct_contains_nested_arr from sub ) sub2; ---- RESULTS -1,'[{"inner_struct":{"str":"","l":0},"small":2},null,{"inner_struct":{"str":"some_string","l":5},"small":20}]','[["2022-12-05","2022-12-06",null,"2022-12-07"],["2022-12-08","2022-12-09",null]]',2 -2,'[{"inner_struct":null,"small":104},{"inner_struct":{"str":"aaa","l":28},"small":105},null]','[["2022-12-10","2022-12-11",null,"2022-12-12"],null]',2754 +1,'[{"inner_struct":{"str":"","l":0},"small":2},null,{"inner_struct":{"str":"some spaceship captain","l":5},"small":20}]','[["2022-12-05","2022-12-06",null,"2022-12-07"],["2022-12-08","2022-12-09",null]]',2 +2,'[{"inner_struct":null,"small":104},{"inner_struct":{"str":"a few soju distilleries","l":28},"small":105},null]','[["2022-12-10","2022-12-11",null,"2022-12-12"],null]',2754 ---- TYPES INT,STRING,STRING,INT ==== @@ -542,9 +542,9 @@ from collection_struct_mix, collection_struct_mix.arr_contains_nested_struct arr ---- RESULTS 1,'{"str":"","l":0}',2 1,'NULL',NULL -1,'{"str":"some_string","l":5}',20 +1,'{"str":"some spaceship captain","l":5}',20 2,'NULL',104 -2,'{"str":"aaa","l":28}',105 +2,'{"str":"a few soju distilleries","l":28}',105 2,'NULL',NULL ---- TYPES INT,STRING,SMALLINT diff --git a/testdata/workloads/functional-query/queries/QueryTest/nested-array-in-select-list.test b/testdata/workloads/functional-query/queries/QueryTest/nested-array-in-select-list.test index 1793c62b2..ca084c3eb 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/nested-array-in-select-list.test +++ b/testdata/workloads/functional-query/queries/QueryTest/nested-array-in-select-list.test @@ -434,7 +434,9 @@ INT,INT,STRING # IMPALA-11434: "More than 1 2d arrays in select list causes analysis error" select id, arr_int_1d, arr_int_2d, arr_int_3d, arr_string_1d, arr_string_2d, arr_string_3d from collection_tbl; ---- RESULTS -1,'[1,2,null]','[[1,2,null],[3]]','[[[1,2,null],[3]],[[4]]]','["1","2",null]','[["1","2",null],["3"]]','[[["1","2",null],["3"]],[["4"]]]' +1,'[1,2,null]','[[1,2,null],[3]]','[[[1,2,null],[3]],[[4]]]','["1","two wooden boxes",null]','[["one silk glove","2",null],["three pancakes"]]','[[["1","second harmonic",null],["three cities"]],[["four castles"]]]' +2,'[1,null,3]','[[null,1,2,null],[5,14,null]]','[[[null,1,2,null],[5,14,null]],[[null,5]]]','["one dinosaur bone",null,"2",null]','[["1","2",null,"four dinosaur bones"],["five dinosaur bones"]]','[[["second dinosaur bone",null,null],["three dinosaur bones"]],[["one",null,"four dinosaur bones"]]]' +3,'[null,4679,null,49,null]','[[1,2,null,null,856],[365,855,369,null]]','[[[1,null,2,null],[null,15]],[[null,4]]]','["1",null,"three even-toed ungulates"]','[["one even-toed ungulate","2",null,null],[null,"three even-toed ungulates"]]','[[["1","-1","second even-toed ungulate",null],["three even-toed ungulates"]],[["four even-toed ungulate"]]]' ---- TYPES INT,STRING,STRING,STRING,STRING,STRING,STRING ==== @@ -446,9 +448,9 @@ from alltypestiny a left outer join (select id, arr_int_1d, arr_int_2d from collection_tbl) b on a.id = b.id where a.id < 3; ---- RESULTS -0,'NULL','NULL' 1,'[1,2,null]','[[1,2,null],[3]]' -2,'NULL','NULL' +2,'[1,null,3]','[[null,1,2,null],[5,14,null]]' +0,'NULL','NULL' ---- TYPES INT,STRING,STRING ===== diff --git a/testdata/workloads/functional-query/queries/QueryTest/nested-map-in-select-list.test b/testdata/workloads/functional-query/queries/QueryTest/nested-map-in-select-list.test index 114b87009..0f379db1e 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/nested-map-in-select-list.test +++ b/testdata/workloads/functional-query/queries/QueryTest/nested-map-in-select-list.test @@ -361,7 +361,9 @@ BIGINT,STRING,INT # IMPALA-11434: "More than 1 2d arrays in select list causes analysis error" select id, map_1d, map_2d, map_3d, arr_int_3d, map_map_array from collection_tbl; ---- RESULTS -1,'{1:"first",2:"second"}','{1:{10:"ten",20:"twenty"},2:{30:"thirty",40:"forty"}}','{1:{10:{100:"hundred",200:"two hundred"},20:{300:"three hundred",400:"four hundred"}},2:{30:{500:"five hundred",600:"six hundred"},40:{700:"seven hundred",800:"eight hundred"}}}','[[[1,2,null],[3]],[[4]]]','{1:{10:[100,200],20:[300,400]},2:{30:[500,600],40:[700,800]}}' +1,'{1:"first automobile",2:"second"}','{1:{10:"ten",20:"twentieth paragraph"},2:{30:"thirty minutes",40:"forty"}}','{1:{10:{100:"hundred",200:"two hundred pages"},20:{300:"three hundred pages",400:"four hundred"}},2:{30:{500:"five hundred pages",600:"six hundred"},40:{700:"seven hundred pages",800:"eight hundred"}}}','[[[1,2,null],[3]],[[4]]]','{1:{10:[100,200],20:[300,400]},2:{30:[500,600],40:[700,800]}}' +2,'{1:"first dinosaur bone",2:"second",3:null}','{1:{10:"ten dinosaur bones",20:"20"},2:{30:"thirty dinosaur bones",40:"forty dinosaur bones"}}','{1:{10:{100:"hundred",200:"two hundred dinosaur bones"},20:{300:"three hundred dinosaur bones",400:"four hundred"}},2:{30:{500:"five hundred dinosaur bones",600:"six hundred"},40:{700:"seven hundred dinosaur bones",800:"eight hundred"}}}','[[[null,1,2,null],[5,14,null]],[[null,5]]]','{1:{10:[100,200],20:[300,400]},2:{30:[500,600],40:[700,800]}}' +3,'{645:"fourth even-toed ungulate",5:"fifth"}','{1:{10:"ten",20:"twentieth even-toed ungulate"},2:{30:"thirty even-toed ungulates",40:"forty"}}','{1:{10:{100:"hundred",200:"two hundred even-toed ungulates"},20:{300:"three hundred even-toed ungulates",400:"four hundred"}},2:{30:{500:"five hundred even-toed ungulates",600:"six hundred"},40:{700:"seven hundred even-toed ungulates",800:"eight hundred"}}}','[[[1,null,2,null],[null,15]],[[null,4]]]','{1:{10:[100,200],20:[300,400]},2:{30:[500, [...] ---- TYPES INT,STRING,STRING,STRING,STRING,STRING ===== @@ -369,10 +371,18 @@ INT,STRING,STRING,STRING,STRING,STRING select id, map_1d, map_2d, mma.value mma_value, ma.value ma_value from collection_tbl c, c.map_map_array mma, mma.value ma; ---- RESULTS -1,'{1:"first",2:"second"}','{1:{10:"ten",20:"twenty"},2:{30:"thirty",40:"forty"}}','{10:[100,200],20:[300,400]}','[100,200]' -1,'{1:"first",2:"second"}','{1:{10:"ten",20:"twenty"},2:{30:"thirty",40:"forty"}}','{10:[100,200],20:[300,400]}','[300,400]' -1,'{1:"first",2:"second"}','{1:{10:"ten",20:"twenty"},2:{30:"thirty",40:"forty"}}','{30:[500,600],40:[700,800]}','[500,600]' -1,'{1:"first",2:"second"}','{1:{10:"ten",20:"twenty"},2:{30:"thirty",40:"forty"}}','{30:[500,600],40:[700,800]}','[700,800]' +1,'{1:"first automobile",2:"second"}','{1:{10:"ten",20:"twentieth paragraph"},2:{30:"thirty minutes",40:"forty"}}','{10:[100,200],20:[300,400]}','[100,200]' +1,'{1:"first automobile",2:"second"}','{1:{10:"ten",20:"twentieth paragraph"},2:{30:"thirty minutes",40:"forty"}}','{10:[100,200],20:[300,400]}','[300,400]' +1,'{1:"first automobile",2:"second"}','{1:{10:"ten",20:"twentieth paragraph"},2:{30:"thirty minutes",40:"forty"}}','{30:[500,600],40:[700,800]}','[500,600]' +1,'{1:"first automobile",2:"second"}','{1:{10:"ten",20:"twentieth paragraph"},2:{30:"thirty minutes",40:"forty"}}','{30:[500,600],40:[700,800]}','[700,800]' +2,'{1:"first dinosaur bone",2:"second",3:null}','{1:{10:"ten dinosaur bones",20:"20"},2:{30:"thirty dinosaur bones",40:"forty dinosaur bones"}}','{10:[100,200],20:[300,400]}','[100,200]' +2,'{1:"first dinosaur bone",2:"second",3:null}','{1:{10:"ten dinosaur bones",20:"20"},2:{30:"thirty dinosaur bones",40:"forty dinosaur bones"}}','{10:[100,200],20:[300,400]}','[300,400]' +2,'{1:"first dinosaur bone",2:"second",3:null}','{1:{10:"ten dinosaur bones",20:"20"},2:{30:"thirty dinosaur bones",40:"forty dinosaur bones"}}','{30:[500,600],40:[700,800]}','[500,600]' +2,'{1:"first dinosaur bone",2:"second",3:null}','{1:{10:"ten dinosaur bones",20:"20"},2:{30:"thirty dinosaur bones",40:"forty dinosaur bones"}}','{30:[500,600],40:[700,800]}','[700,800]' +3,'{645:"fourth even-toed ungulate",5:"fifth"}','{1:{10:"ten",20:"twentieth even-toed ungulate"},2:{30:"thirty even-toed ungulates",40:"forty"}}','{10:[100,200],20:[300,400]}','[100,200]' +3,'{645:"fourth even-toed ungulate",5:"fifth"}','{1:{10:"ten",20:"twentieth even-toed ungulate"},2:{30:"thirty even-toed ungulates",40:"forty"}}','{10:[100,200],20:[300,400]}','[300,400]' +3,'{645:"fourth even-toed ungulate",5:"fifth"}','{1:{10:"ten",20:"twentieth even-toed ungulate"},2:{30:"thirty even-toed ungulates",40:"forty"}}','{30:[500,600],40:[700,800]}','[500,600]' +3,'{645:"fourth even-toed ungulate",5:"fifth"}','{1:{10:"ten",20:"twentieth even-toed ungulate"},2:{30:"thirty even-toed ungulates",40:"forty"}}','{30:[500,600],40:[700,800]}','[700,800]' ---- TYPES INT,STRING,STRING,STRING,STRING ===== @@ -394,7 +404,9 @@ select map_date_key from collection_tbl; ---- RESULTS -'{true:"true",false:"false"}','{-1:"a",0:"b",1:"c"}','{-1:"a",0:"b",1:"c"}','{-1:"a",0:"b",1:"c"}','{-1.5:"a",0.25:"b",1.75:"c"}','{-1.5:"a",0.25:"b",1.75:"c"}','{-1.8:"a",0.2:"b",1.2:"c"}','{"one":1,"two":2,"three":3}','{"Mon":1,"Tue":2,"Wed":3,"Thu":4,"Fri":5,"Sat":6,"Sun":7}','{"a":"A","ab":"AB","abc":"ABC"}','{"2022-12-10 08:15:12":"Saturday morning","2022-12-09 18:15:12":"Friday evening"}','{"2022-12-10":"Saturday","2022-12-09":"Friday"}' +'{true:"true",false:"false statement"}','{-1:"a nice sunny day",0:"best day in my life",1:"c"}','{-1:"a nice sunny day",0:"best day in my life",1:"c"}','{-1:"a nice sunny day",0:"best day in my life",1:"c"}','{-1.5:"a nice sunny day",0.25:"best day in my life",1.75:"c"}','{-1.5:"a nice sunny day",0.25:"best day in my life",1.75:"c"}','{-1.8:"a nice sunny day",0.2:"best day in my life",1.2:"c"}','{"one":1,"two":2,"three distinct values":3}','{"Mon":1,"Tue":2,"Wed":3,"Thu":4,"Fri":5,"Sat": [...] +'{true:"true",false:"false dinosaur bones"}','{-1:"a nice dinosaur bone",0:"best dinosaur bone",1:"c"}','{-1:"a nice dinosaur bone",0:"best dinosaur bone",1:"c"}','{-1:"a nice dinosaur bone",0:"best dinosaur bone",1:"c"}','{-1.5:"a nice dinosaur bone",0.25:"best dinosaur bone",1.75:"c"}','{-1.5:"a nice dinosaur bone",0.25:"best dinosaur bone",1.75:"c"}','{-1.8:"a nice dinosaur bone",0.2:"best dinosaur bone",1.2:"c"}','{"one":1,"two":2,"three distinct dinosaur bones":3}','{"Mon":1,"Tue":2 [...] +'{true:"true even-toed ungulate",false:"false"}','{-1:"a nice even-toed ungulate",0:"best even-toed ungulate",1:"c"}','{-1:"a nice even-toed ungulate",0:"best even-toed ungulate",1:"c"}','{-1:"a nice even-toed ungulate",0:"best even-toed ungulate",1:"c"}','{-1.5:"a nice even-toed ungulate",0.25:"best even-toed ungulate",1.75:"c"}','{-1.5:"a nice even-toed ungulate",0.25:"best even-toed ungulate",1.75:"c"}','{-1.8:"a nice even-toed ungulate",0.2:"best even-toed ungulate",1.2:"c"}','{"one" [...] ---- TYPES STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING ===== @@ -417,7 +429,9 @@ select map_date_key from collection_tbl; ---- RESULTS -'{"true":"true","false":"false"}','{"-1":"a","0":"b","1":"c"}','{"-1":"a","0":"b","1":"c"}','{"-1":"a","0":"b","1":"c"}','{"-1.5":"a","0.25":"b","1.75":"c"}','{"-1.5":"a","0.25":"b","1.75":"c"}','{"-1.8":"a","0.2":"b","1.2":"c"}','{"one":1,"two":2,"three":3}','{"Mon":1,"Tue":2,"Wed":3,"Thu":4,"Fri":5,"Sat":6,"Sun":7}','{"a":"A","ab":"AB","abc":"ABC"}','{"2022-12-10 08:15:12":"Saturday morning","2022-12-09 18:15:12":"Friday evening"}','{"2022-12-10":"Saturday","2022-12-09":"Friday"}' +'{"true":"true","false":"false statement"}','{"-1":"a nice sunny day","0":"best day in my life","1":"c"}','{"-1":"a nice sunny day","0":"best day in my life","1":"c"}','{"-1":"a nice sunny day","0":"best day in my life","1":"c"}','{"-1.5":"a nice sunny day","0.25":"best day in my life","1.75":"c"}','{"-1.5":"a nice sunny day","0.25":"best day in my life","1.75":"c"}','{"-1.8":"a nice sunny day","0.2":"best day in my life","1.2":"c"}','{"one":1,"two":2,"three distinct values":3}','{"Mon": [...] +'{"true":"true","false":"false dinosaur bones"}','{"-1":"a nice dinosaur bone","0":"best dinosaur bone","1":"c"}','{"-1":"a nice dinosaur bone","0":"best dinosaur bone","1":"c"}','{"-1":"a nice dinosaur bone","0":"best dinosaur bone","1":"c"}','{"-1.5":"a nice dinosaur bone","0.25":"best dinosaur bone","1.75":"c"}','{"-1.5":"a nice dinosaur bone","0.25":"best dinosaur bone","1.75":"c"}','{"-1.8":"a nice dinosaur bone","0.2":"best dinosaur bone","1.2":"c"}','{"one":1,"two":2,"three distin [...] +'{"true":"true even-toed ungulate","false":"false"}','{"-1":"a nice even-toed ungulate","0":"best even-toed ungulate","1":"c"}','{"-1":"a nice even-toed ungulate","0":"best even-toed ungulate","1":"c"}','{"-1":"a nice even-toed ungulate","0":"best even-toed ungulate","1":"c"}','{"-1.5":"a nice even-toed ungulate","0.25":"best even-toed ungulate","1.75":"c"}','{"-1.5":"a nice even-toed ungulate","0.25":"best even-toed ungulate","1.75":"c"}','{"-1.8":"a nice even-toed ungulate","0.2":"best [...] ---- TYPES STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING =====
